<?php
	/*
	 * used mysql table:
	 * 
		CREATE TABLE `cms_indexcrawler` (
		  `id` int(11) NOT NULL AUTO_INCREMENT,
		  `link` varchar(255) NOT NULL DEFAULT '',
		  `body` longtext,
		  `title` varchar(255) NOT NULL,
		  `last_indexed` varchar(255) NOT NULL,
		  `checked` tinyint(4) NOT NULL DEFAULT '0',
		  PRIMARY KEY (`id`)
		) ENGINE=MyISAM;
	 */

	@chdir('httpdocs');
	
	ini_set('display_errors', 1);
	ini_set('allow_url_fopen', 1);
	ini_set('allow_url_include', 1);

	error_reporting(E_ALL);

	/*
	 * code needed if no inclusion of site_config.php is possible:
	 * 	
	 	Class SiteConfig 
	 	{
		const Domain = 'http://127.0.0.1/aptana/avancecms/dev/'; // do not forget the trailing slash!
		
		const SqlHost = 'localhost';
		
		const SqlUser = 'root';
		
		const SqlPass = 'defaultPassword';
		
		const SqlDatabase = 'framework';
		}
	 */
	
	session_start();
	
	// files need this defined token so they can't be open by outside people.
	define('YMT', true);
	define('YOMM_PATH', 'Yomm/');
	
	require_once('site_config.php');
	require_once(YOMM_PATH . 'Loader.class.php');
	
	$yomm_loader = new Yomm_Loader();
	$yomm_loader->start();

	// Log informations
	$timestamp = mktime();
	
	$folder = logCreationFolder($timestamp);
	
	function logCreationFolder($timestamp)
	{
		$folder = 'storage/';
		if(!file_exists($folder))
		{
			$folder = 'httpdocs/storage/';
			if(!file_exists($folder))
			{
				echo "folder 'storage/' or 'httpdocs/storage/' does not exists! can not write log informations. Abort script by exit now!";
				exit;
			}
		}
		
		if(!is_writable($folder))
		{
			echo 'the folder "' . $folder . '" is not writeable!';
			exit;
		}
		
		return $folder;
	}
	
	if(!function_exists('curl_init'))
	{
		echo "curl is not available on this server! Abort script by exit now!";
		exit;		
	}
	
	/**
	 * website index crawler class
	 * try to find all links and the following links of it; save the body informations in a table to generate google sitemap
	 * or an internal search engine possibility.
	 * @author bsuter
	 */
	class indexCrawler 
	{
		
		private $db = null;
		
		private $tmp_content = null;
		
		private $tmp_title = null;
		
		private $base_host = null;
		
		private $timestamp = null;
		
		private $folder = null;
		
		private $stats = array();
		
		public $log = array();
		
		public function __construct($timestamp, $folder)
		{
			$this->timestamp = $timestamp;
			
			$this->folder = $folder;
			
			$this->db = new Yomm_Db_Query();
			
			$this->db->connect(SiteConfig::SqlHost, SiteConfig::SqlUser, SiteConfig::SqlPass, SiteConfig::SqlDatabase);
			
			$this->stats = array(
				'index_nums_before_exec' => 0,
				'index_nums_after_exec' => 0,
				'links_found_all' => 0,
				'links_found_valid' => 0,
				'empty_pages' => 0
			);
			
			$this->setLog('#! <crawl>');
		}
		
		/**
		 * start crawling from the host url (base url) 
		 */
		public function mainInit($host_url)
		{
			$this->setLog("*info* start crawling url: {$host_url}");
			
			$this->base_host = parse_url($host_url, PHP_URL_HOST);
			
			// set all fields "checked" to false (0)
			$this->db->handler("cms_indexcrawler", array(
				'checked' => 0
			), "WHERE checked='1'");
			
			// list all elements
			$this->stats['index_nums_before_exec'] = $this->db->select("cms_indexcrawler", array("COUNT(*)" => "count"), false, false)->count;
			
			// delete content with null values
			$this->db->deleteSql("cms_indexcrawler", "body is NULL");
			
			$this->setLog("*info* all fields are now unchecked and containt the value 0");
			
			$this->crawlPage($host_url);
			
			$this->findAllNotChecked();
		}
		
		private function findAllNotChecked()
		{
			$select = $this->db->select("cms_indexcrawler", "link", "WHERE checked='0'", true);
			
			foreach($select as $k => $v)
			{
				$this->crawlPage($v->link);
				sleep(2);
			}
			
			$after_select = $this->db->select("cms_indexcrawler", "link", "WHERE checked='0'", true);
			
			if(count($after_select) > 0)
			{
				$this->findAllNotChecked();
			} else {
				$this->setLog("*info* finished crawling not checked url's");
				
				/* FINISH PROCESS */
				
				// log file select all indexed pages
				$this->stats['index_nums_after_exec'] = $this->db->select("cms_indexcrawler", array("COUNT(*)" => "count"), false, false)->count;
				
				// find empty pages
				$this->stats['empty_pages'] = $this->db->select("cms_indexcrawler", array("COUNT(*)" => "count"), "WHERE body is NULL", false)->count;
				
				// clean them up
				$this->db->deleteSql("cms_indexcrawler", "body is NULL");
				
			}
		}
		
		private function crawlPage($url)
		{
			$content = $this->getContentInformations($url);
			
			if(!$this->regexContent())
			{
				// now we can set this page as checked:
				$this->db->handler("cms_indexcrawler", array(
					"checked" => 1
				), "WHERE link='{$url}'");
				return false;
			}
			
			$this->findLinks();
			
			$body = $this->prepareBody($this->tmp_content);
			
			$title = htmlentities(strip_tags($this->tmp_title), ENT_QUOTES);
			// update content if url already existst or insert if not:
			$row = $this->db->select("cms_indexcrawler", "link", "WHERE link='{$url}'", false);
			
			if(count($row) == 0)
			{
				// insert new
				$this->db->handler("cms_indexcrawler", array(
					"link" => $url,
					"body" => $body,
					"title" => $title,
					"last_indexed" => date("d.m.Y H:i:s")
				));
				$this->setLog("<== {$url} was successfully inserted!");
			} else {
				// update existing
				if(count($row) > 1)
				{
					$this->setLog("something went wrong. there are to much entrys (".count($row).") for url: \"{$url}\"");
				} else {
					$this->db->handler("cms_indexcrawler", array(
						"link" => $url,
						"body" => $body,
						"title" => $title,
						"last_indexed" => date("d.m.Y H:i:s")
					), "WHERE link='{$row->link}'");
					$this->setLog("<== {$url} was successfully updated!");
				}
			}
			
			// now we can set this page as checked:
			$this->db->handler("cms_indexcrawler", array(
				"checked" => 1
			), "WHERE link='{$url}'");
			
			return true;
		}
		
		private function prepareBody($body)
		{
			preg_match_all("/\[CRAWL_IGNORE\](.*?)\[\/CRAWL_IGNORE\]/s", $body, $output);
			$body = str_replace($output[0], '', $body);
			
			// delete script tags
			preg_match_all("/\<script(.*?)\<\/script\>/s", $body, $output);
			$body = str_replace($output[0], '', $body);
			
			// delete html comments at all
			preg_match_all("/\<\!\-\-(.*?)\-\-\>/s", $body, $output);
			$body = str_replace($output[0], '', $body);
			
			$body = htmlentities(strip_tags($body), ENT_QUOTES);
			
			return $body;
		}
		
		private function getContentInformations($url)
		{
			$this->setLog("==> crawl content from: {$url}");
				
			$ch = curl_init();
			curl_setopt($ch, CURLOPT_URL, $url);
			curl_setopt($ch, CURLOPT_HEADER, false);
			curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
			$content = curl_exec($ch);
			curl_close($ch);
			
			$this->tmp_content = $content;
			
			return true;
		}
		
		private function regexContent()
		{
			preg_match('/\<body\>(.*?)\<\/body\>/s', $this->tmp_content, $result); // using (.*?) OR (.*) ?
	
			// find everything between <body></body>
			if(count($result) == 0)
			{
				$this->setLog("*alert* content does not contain body tags.");
				return false;
			}
			
			// find title between <title></title>
			preg_match('/\<title\>(.*?)\<\/title\>/s', $this->tmp_content, $result2);
			
			if(count($result2) == 0)
			{
				$this->setLog("*alert* content does not have a title tag.");
				return false;
			}
			
			$this->tmp_content = $result[1];
			
			$this->tmp_title = $result2[1];
			
			return true;
		}
		
		/**
		 * try to find links with html or trailing slash .. but not images, js or ss
		 */
		private function findLinks()
		{
			preg_match_all('/href=[\"\']([a-zA-Z0-9\:\/\.\-\_\?\=\&]+)[\"\']/im', $this->tmp_content, $result);
			
			foreach($result[1] as $link)
			{
				
				if($this->base_host !== parse_url($link, PHP_URL_HOST)) continue; // its not the same host! we don't want to crawl extranl pages
				
				$this->stats['links_found_all'] = $this->stats['links_found_all'] + 1;
				
				/*
				 * read the headers with curl instead of get_headers();
				 * => $h = get_headers($link, 1);
				*/
				
				$ch = curl_init($link);
				curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
				curl_exec($ch);
			
				if(!curl_errno($ch))
				{
				 	$content_type = curl_getinfo($ch, CURLINFO_CONTENT_TYPE);
					//$content_type = $info['content_type'];
				} else {
					$this->setLog('*error* curl error while getting header of: ' . $link);
					continue;
				}
				
				curl_close($ch);
				
				/*
				 * EOF CURL
				 */
				if($content_type !== "text/html") continue; // lets see if its a valid header (text content)
				
				$this->stats['links_found_valid'] = $this->stats['links_found_valid'] + 1;
				
				// check if existing in database
				$check = $this->db->select("cms_indexcrawler", "checked", "WHERE link='{$link}'", false);
				if(count($check) == 0) // link does not exists - add as new!
				{
					$add = $this->db->handler("cms_indexcrawler", array(
						"link" => $link
					));
					
					$this->setLog("*info* added link '{$link}' to table which will be scanned soon.");
				}
			}
			
			return true;
		}
		
		public function getStats()
		{
			/* internal-stats-count-array:
			 	array(
				  	'index_nums_before_exec' => 0,
					'index_nums_after_exec' => 0,
					'links_found_all' => 0,
					'links_found_valid' => 0,
					'empty_pages' => 0
				);
			 */
			$stats = "------------ STATISTIC ------------\n";
			$stats.= "Pages befor exec: {$this->stats['index_nums_before_exec']}\n";
			$stats.= "Pages after exec: {$this->stats['index_nums_after_exec']}\n";
			$stats.= "New Pages: ". ($this->stats['index_nums_after_exec'] - $this->stats['index_nums_before_exec']) ."\n";
			$stats.= "All found Links: {$this->stats['links_found_all']}\n";
			$stats.= "All links with valid header: {$this->stats['links_found_valid']}\n";
			$stats.= "Deleted empty/null Pages: {$this->stats['empty_pages']}\n";
			$stats.= "Duration: ". round(((mktime() - $this->timestamp)/60),2) . " min\n";
			$stats.= "-----------------------------------";
			
			return $stats;
		}
		
		public function __destruct()
		{
			$this->setLog($this->getStats());
			$this->setLog('!# </crawl>');
		}
		
		private function setLog($text)
		{
			$text = date("d.m.Y H:i:s") . ": {$text}\n";
			$file = $this->folder . "crawlerlog_{$this->timestamp}.txt";
			$fh = fopen($file, 'a+') or die("can't open file");
			fwrite($fh, $text);
		}
	}
	
	// start the crawler
	$crawl = new indexCrawler($timestamp, $folder);
	// execute the main crawler function
	$crawl->mainInit(SiteConfig::Domain);
	
	// print success if nothing happend
	echo "crawler.php execution is done. See the log file for more informations " . SiteConfig::Domain . str_replace("httpdocs/", "", $folder) . "crawlerlog_{$timestamp}.txt\n\n" . $crawl->getStats();
	
	unset($crawl);